package com.barry.crawler;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class WebUtil {
    public static String getPageContent(String url){
        StringBuffer sb = new StringBuffer();
        try {
            // 建立连接
            URL u = new URL(url);
            HttpURLConnection httpUrlConn = (HttpURLConnection) u.openConnection();
            httpUrlConn.setDoInput(true);
            httpUrlConn.setRequestMethod("GET");
            //设置请求头
//            httpUrlConn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
//            httpUrlConn.setRequestProperty("Accept-Encoding","gzip, deflate, br");
//            httpUrlConn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
//            httpUrlConn.setRequestProperty("Connection", "keep-alive");
            //            httpUrlConn.setRequestProperty("Host", "www.tianyancha.com");
//            httpUrlConn.setRequestProperty("Referer", "https://www.tianyancha.com/");
//            httpUrlConn.setRequestProperty("Upgrade-Insecure-Requests", "1");
            httpUrlConn.setRequestProperty("Cookie","小点心");
            httpUrlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36");
            // 获取输入流
            InputStream is = httpUrlConn.getInputStream();
            // 将字节输入流转换为字符输入流
            InputStreamReader isr = new InputStreamReader(is, "utf-8");
            // 为字符输入流添加缓冲
            BufferedReader br = new BufferedReader(isr);
            // 读取返回结果
            String data = null;
            while ((data = br.readLine()) != null) {
                sb.append(data);
//                System.out.println(data);
            }
            // 释放资源
            br.close();
            isr.close();
            is.close();
            httpUrlConn.disconnect();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sb.toString();
    }
}